#!/usr/local/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/9/17 9:50
# @Author  : Kyy
# @Site    : 
# @File    : robo.py
# @Software: PyCharm

import requests
import time
import threading
from connmongo import MongoConn
from pymongo.errors import DuplicateKeyError

class robo1():
    def __init__(self,stockid):
        self.stockid = stockid
        self.headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36'
                        ,'Cookie':'cloud-anonymous-token=432955f1f53042089c075d3692b6d209; _ga=GA1.2.958565960.1553238451; _gid=GA1.2.1446556807.1553238451; UM_distinctid=169a439e45b96-0ade27a39e0e63-9333061-144000-169a439e45c22f; gr_user_id=23f0fbc9-dbde-4375-bcad-1d61866d8a8d; grwng_uid=2181b16e-c9d7-449a-90e1-274aae8d698b; _DA_pingback=01396f03-62f0-4ebe-806b-caa582a854dd; cloud-sso-token=9009091DECB52A5D287FA0AA85D68AC4; ba895d61f7404b76_gr_session_id=6920757a-8b8d-43a2-ab80-4491e8fb8c0e; ba895d61f7404b76_gr_session_id_6920757a-8b8d-43a2-ab80-4491e8fb8c0e=true; CNZZDATA1257961942=1745220253-1553233472-https%253A%252F%252Fwww.baidu.com%252F%7C1553244279; _gat=1'}
        self.proxy = {'https': 'http://60.191.57.79:10800'}

    # 获取股票中文名
    def getname(self):
        res = self.getres1(inurl="https://gw.datayes.com/rrp/web/search/stockInfo")
        if res:
            return res['shortNM']

    # 股票基本信息,盈利指标预测,经营数据预测,盈利预测分析,近一年同业公司对比估值
    def getres1(self,inurl,freq=0):
        if freq == 0:
            payload = {'ticker':self.stockid}
        else:
            payload = {'ticker':self.stockid,'freq':freq}
        res = requests.get(inurl,headers=self.headers,params=payload)
        if res.status_code == 200:
            return res.json()['data']

    # 主营构成,新闻
    def getres2(self,inurl):
        url = inurl + str(self.stockid)
        res = requests.get(url,headers=self.headers)
        if res.status_code == 200:
            return res.json()['data']

    # 重要公告
    def getres3(self,inurl,pagesize):
        res = requests.post(inurl,headers=self.headers,json={'ticker':self.stockid,'pageSize':pagesize,'pageNow':1,'important':1})
        res.encoding = 'utf-8'
        if res.status_code == 200:
            return res.json()['data']

    # 券商研报
    def getres4(self,inurl,pagesize):
        name = self.getname()
        payload = {'query':name,'pageSize':pagesize,'pageNow':1,'type':'EXTERNAL_REPORT'}
        res = requests.get(inurl,headers=self.headers,params=payload)
        if res.status_code == 200:
            return res.json()['data']

    def runall(self):
        result = {}
        result['股票代码'] = self.stockid
        result['股票基本信息'] = self.getres1(inurl="https://gw.datayes.com/rrp/web/search/stockInfo")
        result['主营构成'] = self.getres2(inurl="https://gw.datayes.com/rrp/web/stockModelSummary/mainOperationData/")
        result['盈利指标预测'] = self.getres1(inurl="https://gw.datayes.com/rrp/web/stockModelSummary/finPrediction",freq=3)
        result['经营数据预测'] = self.getres1(inurl="https://gw.datayes.com/rrp/web/stockModelSummary/mainOperPrediction",freq=3)
        result['盈利预测分析'] = self.getres1(inurl="https://gw.datayes.com/rrp/web/stockModelSummary/summary")
        result['近一年同业公司对比估值'] = self.getres1(inurl="https://gw.datayes.com/rrp/web/stock/comparison/report/table")
        result['新闻'] = self.getres2(inurl="https://gw.datayes.com/rrp/web/news/ticker/")
        result['券商研报'] = self.getres4(inurl="https://gw.datayes.com/rrp/web/search",pagesize=30)
        result['重要公告'] = self.getres3(inurl="https://gw.datayes.com/rrp/web/getAnnouncementsByTicker",pagesize=30)
        return result

with open('stock.txt','r') as f:
    stocks = f.read()
stockslst = stocks.split('/n')
# print(stockslst)
stockcode = [eval(i)[0] for i in stockslst if i]
print(stockcode)
print(len(stockcode))

successnum = 0

def getrst(stocklst):
    my_conn = MongoConn()
    rst = []
    for it in stocklst:
        test = robo1(stockid=it)
        res = test.runall()
        rst.append(res)
        try:
            my_conn.db['robo'].insert_one(res)
            global successnum
            successnum += 1
        except DuplicateKeyError:
            pass
        print("已经有{}条数据".format(successnum))

threads = []
nums = int(len(stockcode)/50) + 1
for i in range(nums):
    t = threading.Thread(target=getrst,args=(stockcode[i*50:(i+1)*50],))
    threads.append(t)

for i in range(nums):
    threads[i].start()

for i in range(nums):
    threads[i].join()

print("Done!")







# "股票基本信息": https://gw.datayes.com/rrp/web/search/stockInfo?ticker=600115
# "主营构成" ： https://gw.datayes.com/rrp/web/stockModelSummary/mainOperationData/600115
# "盈利指标预测": https://gw.datayes.com/rrp/web/stockModelSummary/finPrediction?ticker=600115&freq=3
# "经营数据预测": https://gw.datayes.com/rrp/web/stockModelSummary/mainOperPrediction?ticker=600115&freq=3
# "盈利预测分析": https://gw.datayes.com/rrp/web/stockModelSummary/summary?ticker=600115
# "近一年同业公司对比估值": https://gw.datayes.com/rrp/web/stock/comparison/report/table?ticker=600115
# "新闻" ： https://gw.datayes.com/rrp/web/news/ticker/600115
# "券商研报"：https://gw.datayes.com/rrp/web/search?query=%E4%B8%9C%E6%96%B9%E8%88%AA%E7%A9%BA&pageNow=1&pageSize=30&type=EXTERNAL_REPORT
# "重要公告":https://gw.datayes.com/rrp/web/getAnnouncementsByTicker



# UM_distinctid=165e5309553151-083af4b2f54149-5701732-144000-165e5309555230; _ga=GA1.2.35328657.1537148491; _gid=GA1.2.1224869340.1537148491; _DA_pingback=9e2609c9-2fcc-4967-8dfb-274eba710ff5; _gat=1; cloud-anonymous-token=2f977c1a168944e0b63f587ededf5417; cloud-sso-token=59AD40224A48374C0667610C6F4311F6; CNZZDATA1257961942=2000903870-1537146581-%7C1537229109
# UM_distinctid=165e5309553151-083af4b2f54149-5701732-144000-165e5309555230; _ga=GA1.2.35328657.1537148491; _gid=GA1.2.1224869340.1537148491; cloud-promotion-project=rrp; cloud-promotion-method=online; cloud-promotion-channel=%E5%BE%AE%E4%BF%A1MP-weixinmp; cloud-promotion-id=299; cloud-anonymous-token=879d7164331d4d2eac05d5153134896b; cloud-sso-token=F93D9E8AD970476B519A79D67DDF1FF2; JSESSIONID=DAD5E55711EF50C1240239003140BE40
# UM_distinctid=165e5309553151-083af4b2f54149-5701732-144000-165e5309555230; _ga=GA1.2.35328657.1537148491; _gid=GA1.2.1224869340.1537148491; _DA_pingback=9e2609c9-2fcc-4967-8dfb-274eba710ff5; cloud-anonymous-token=2f977c1a168944e0b63f587ededf5417; cloud-sso-token=59AD40224A48374C0667610C6F4311F6; CNZZDATA1257961942=2000903870-1537146581-%7C1537230823; _gat=1
# UM_distinctid=165e5309553151-083af4b2f54149-5701732-144000-165e5309555230; _ga=GA1.2.35328657.1537148491; _gid=GA1.2.1224869340.1537148491; _DA_pingback=9e2609c9-2fcc-4967-8dfb-274eba710ff5; cloud-anonymous-token=2f977c1a168944e0b63f587ededf5417; cloud-sso-token=59AD40224A48374C0667610C6F4311F6; CNZZDATA1257961942=2000903870-1537146581-%7C1537229109